package com.choudou5.spider.util;

import cn.hutool.core.io.FileUtil;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.choudou5.solr.util.StrUtil;
import com.choudou5.spider.consts.SpiderConsts;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringEscapeUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.math.NumberUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.nodes.Node;
import org.jsoup.nodes.TextNode;
import org.jsoup.safety.Whitelist;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.net.URL;
import java.net.URLEncoder;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * @name 解析数据工具类
 * @author choudou5
 * @date 2018-08-21
 */
public class ParseDataUtil {

    private static Logger logger = LoggerFactory.getLogger(ParseDataUtil.class);

    /**
     * jsoup初步清洗 白名单规则
     *
     * @update choudou5 20180201
     * 1、a标签 兼容相对地址
     * 2、保留id属性  table、tr、td、div、 ul
     */
    public static final Whitelist JSOUP_WHITE_LIST_RELAXED = (new Whitelist()).addTags(new String[]{"a", "b", "blockquote", "br", "caption", "cite", "code", "col", "colgroup", "dd", "div", "dl", "dt", "em", "h1", "h2", "h3", "h4", "h5", "h6", "i", "img", "li", "ol", "p", "pre", "q", "small", "span", "strike", "strong", "sub", "sup", "table", "tbody", "td", "tfoot", "th", "thead", "tr", "u", "ul"})
            .addAttributes("a", new String[]{"href", "title"})
            .addAttributes("blockquote", new String[]{"cite"})
            .addAttributes("col", new String[]{"span", "width"})
            .addAttributes("colgroup", new String[]{"span", "width"})
            .addAttributes("div", new String[]{"id"})
            .addAttributes("img", new String[]{"align", "alt", "height", "src", "title", "width"})
            .addAttributes("ol", new String[]{"start", "type"})
            .addAttributes("q", new String[]{"cite"})
            .addAttributes("table", new String[]{"summary", "width", "id"})
            .addAttributes("tr", new String[]{"id"})
            .addAttributes("td", new String[]{"id", "abbr", "axis", "colspan", "rowspan", "width", "title"})
            .addAttributes("th", new String[]{"abbr", "axis", "colspan", "rowspan", "scope", "width"})
            .addAttributes("ul", new String[]{"type", "id"})
                    //.addProtocols("a", "href", new String[]{"ftp", "http", "https", "mailto"})
            .addProtocols("blockquote", "cite", new String[]{"http", "https"})
            .addProtocols("cite", "cite", new String[]{"http", "https"})
            .addProtocols("img", "src", new String[]{"http", "https"})
            .addProtocols("q", "cite", new String[]{"http", "https"});

    /**
     * 通过正则表达式匹配出字符串并返回
     *
     * @param el     正则表达式
     * @param source 需要匹配的源
     * @return
     */
    public static String matchData(String el, String source) {
        Pattern pattern = Pattern.compile(el);
        Matcher matcher = pattern.matcher(source);
        StringBuffer sbr = new StringBuffer();
        while (matcher.find()) {
            String value = matcher.group();
            if (value.length() < 200) {
//				System.out.println(value);
                sbr.append(value);
                break;
            }
        }
        return sbr.toString();
    }

    /**
     * 通过正则表达式第一个匹配出字符串
     *
     * @param el     正则表达式
     * @param source 需要匹配的源
     * @return
     */
    public static String firstMatchData(String el, String source) {
        Pattern pattern = Pattern.compile(el);
        Matcher matcher = pattern.matcher(source);
        StringBuffer sbr = new StringBuffer();
        if (matcher.find()) {
            return matcher.group(1);
        }
        return sbr.toString();
    }


    /**
     * @param el
     * @param str
     * @return
     */
    public static boolean match(String el, String str) {
        Pattern pattern = Pattern.compile(el);
        Matcher matcher = pattern.matcher(str);
        if (matcher.find()) {
            return true;
        } else {
            return false;
        }
    }

    /**
     * 判断是否为整数
     *
     * @param str 传入的字符串
     * @return 是整数返回true, 否则返回false
     */
    public static boolean isInteger(String str) {
        Pattern pattern = Pattern.compile("^[-\\+]?[\\d]*$");
        return pattern.matcher(str).matches();
    }

    private static final String JSOUP_MARK_THIS_TEXT = "this@text";

    private static String wrapHtml(String html) {
        if (StringUtils.isBlank(html))
            return null;
        if (StringUtils.startsWith(html, "<tbody>")) {
            html = SpiderHtmlUtil.wrapTag(html, "table");
        }
        if (StringUtils.startsWith(html, "<tr")) {
            html = SpiderHtmlUtil.wrapTag(html, "table");
        }
        return html;
    }

    /**
     * 解析 html
     * @param html
     * @return
     */
    public static Element jsoupFormat(String html) {
        if (StringUtils.isBlank(html))
            return null;
        html = wrapHtml(html);
        return Jsoup.parse(html);
    }

    /**
     * 解析 html
     * @param html
     * @return
     */
    public static String jsoupFormatToText(String html) {
        if (StringUtils.isBlank(html))
            return null;
        html = wrapHtml(html);
        return Jsoup.parse(html).text();
    }


    /**
     * 提取标签文本
     * @param html
     * @return
     */
    public static Set<String> jsoupExtractTagText(String html, String tag) {
        if (StringUtils.isBlank(html))
            return null;
        html = wrapHtml(html);
        Element element = Jsoup.parse(html);
        Elements eles = element.select(tag);
        Set<String> result = new HashSet<>();
        if(eles != null && eles.size() > 0){
            for (Element ele : eles) {
                result.add(ele.text());
            }
        }
        return result;
    }


    /**
     * 解析 html
     *
     * @param html
     * @param ruleStr 规则
     * @param extractHtml 提取Html
     * @return
     */
    public static String jsoup(String html, String ruleStr, boolean extractHtml) {
        if (StringUtils.isBlank(html) || StringUtils.isBlank(ruleStr))
            return null;
        html = wrapHtml(html);
        String value = "";
        boolean isGetThisText = false;//获取 当前元素文本
        if (StringUtils.contains(ruleStr, JSOUP_MARK_THIS_TEXT)) {
            String[] array = StringUtils.split(ruleStr, JSOUP_MARK_THIS_TEXT);
            if (array.length == 0) {
                Document doc = Jsoup.parse(html);
                value = getChildTextNodeVal(doc.childNodes(), false);
                return value;
            } else {
                ruleStr = array[0];
                isGetThisText = true;
            }
        }
        String attrTag = "@";
        String attrKey = null;
        if (ruleStr.contains(attrTag)) {
            String[] arr = ruleStr.split(attrTag);
            ruleStr = arr[0];
            attrKey = arr[1];
        }
        ruleStr = jsoupRuleTransfer(ruleStr);
        boolean isGetLast = false;
        if(ruleStr.contains(":last")){
            isGetLast = true;
            ruleStr = StrUtil.subBefore(ruleStr, ":last", true);
        }
        Document doc = Jsoup.parse(html);
        Elements ele = doc.select(ruleStr);
        if(ele.size() == 0)
            return null;
        String cont = null;
        if (isGetThisText) {
            Document doc2 = Jsoup.parse(ele.html());
            value = getChildTextNodeVal(doc2.childNodes(), false);
        }else {
            Element element = isGetLast ? ele.last(): ele.get(0);
            if(attrKey != null){
                value = element.attr(attrKey);
            }else{
                if(extractHtml)
                    cont = element.html();
                else
                    cont = element.text();
                value = StringUtils.isBlank(cont) ? ele.val() : cont;
            }
        }
        return value;
    }

    private static String getChildTextNodeVal(List<Node> nodes, boolean intoEle) {
        if (intoEle) {
            for (Node node : nodes) {
                List<Node> nodeList = node.childNodes();
                if (!nodeList.isEmpty())
                    return getChildTextNodeVal(nodeList, false);
            }
        } else {
            for (Node node : nodes) {
                if (node instanceof TextNode) {
                    return ((TextNode) node).text();
                }
            }
        }
        return getChildTextNodeVal(nodes, true);
    }

    public static String getTextNodeVal(Elements eles) {
        String result = "";
        if (eles != null && !eles.isEmpty()) {
            Iterator<Element> iter = eles.iterator();
            for1:
            while (iter.hasNext()) {
                Element ele = iter.next();
                List<TextNode> list = ele.textNodes();
                if (CollectionUtils.isNotEmpty(list)) {
                    for2:
                    for (TextNode textNode : list) {
                        if (StringUtils.isNotBlank(textNode.text())) {
                            result = textNode.text();
                            break for1;
                        }
                    }
                }
            }
        }
        return result;
    }


    public static int pageCalculate(long totalNum, int pageSize) {
        return (int) ((totalNum / pageSize) + (totalNum % pageSize == 0 ? 0 : 1));
    }

    /**
     * 解析 html
     *
     * @param html
     * @param ruleStr 规则
     * @return
     */
    public static Element jsoupReturnElement(String html, String ruleStr) throws Exception {
        if (StringUtils.isBlank(html))
            return null;
        ruleStr = jsoupRuleTransfer(ruleStr);
        Document doc = Jsoup.parse(html);
        if (StringUtils.isBlank(ruleStr)) {
            return doc.select("body").get(0);
        }
        Elements eles = doc.select(ruleStr);
        if (!eles.isEmpty()) {
            return eles.get(0);
        } else {
//            logger.info(html);
            logger.warn("jsoupReturnElement 解析失败, 规则：" + ruleStr);
            throw new Exception("jsoupReturnElement 解析失败, 规则：" + ruleStr);
        }
    }

    /**
     * 解析 html
     *
     * @param html
     * @param ruleStr 规则
     * @return
     */
    public static Elements jsoupReturnElements(String html, String ruleStr) {
        if (StringUtils.isBlank(html) || StringUtils.isBlank(ruleStr))
            return null;
        ruleStr = jsoupRuleTransfer(ruleStr);
        html = SpiderHtmlUtil.checkSpecialHtmlContent(html); //检查 特殊网页内容
        Document doc = Jsoup.parse(html);
        Elements eles = doc.select(ruleStr);
        if (!eles.isEmpty()) {
            return eles;
        } else {
            logger.info(html);
            logger.warn("jsoupReturnElements 解析失败, 规则：" + ruleStr);
        }
        return null;
    }

    private static String jsoupRuleTransfer(String ruleStr) {
        if (StringUtils.isBlank(ruleStr))
            return null;
        return ruleStr.replace("&gt;", ">").replace("&lt;", "<");
    }

    /**
     * 是否为 json内容
     *
     * @param conetnt
     * @return
     */
    public static boolean isJsonContent(String conetnt) {
        return StringUtils.startsWith(conetnt, "{") || StringUtils.startsWith(conetnt, "[");
    }


    /**
     * 解析 json
     *
     * @param jsonData
     * @param ruleStr  规则
     * @return
     */
    public static String jsonVal(String jsonData, String ruleStr) {
        if (StringUtils.isBlank(jsonData) || StringUtils.isBlank(ruleStr))
            return null;
        String[] rules = ruleStr.split(" ");
        int ruleLen = rules.length;
        JSONObject jsonObject = JSON.parseObject(jsonData);
        String rule = null;
        String result = null;
        for (int i = 1; i <= ruleLen; i++) {
            rule = rules[i - 1];
            if (ruleLen == i) {
                return SpiderJsonUtil.formatJsonVal(jsonObject, rule);
            } else {
                jsonObject = jsonObject.getJSONObject(rule);
            }
        }
        if (result == null) {
            logger.warn("json 解析失败, 规则：" + ruleStr);
        }
        return result;
    }


    /**
     * 解析 json
     *
     * @param jsonObj
     * @param ruleKey 规则
     * @return
     */
    public static String jsonVal(JSONObject jsonObj, String ruleKey) {
        if (jsonObj == null || StringUtils.isBlank(ruleKey))
            return null;
        //是否为拼接字段
        if (SpiderJsonUtil.isJoinField(ruleKey)) {
            return SpiderJsonUtil.getJoinFieldValue(jsonObj, ruleKey);
        }
        String result = SpiderJsonUtil.formatJsonVal(jsonObj, ruleKey);
        if (result == null) {
            logger.warn("jsonVal 解析失败, 规则：" + ruleKey);
        }
        return result;
    }


    /**
     * 解析 json
     *
     * @param jsonData
     * @return
     */
    public static JSONArray jsonArray(String jsonData) {
        if (StringUtils.isBlank(jsonData))
            return null;
        return JSON.parseArray(jsonData);
    }

    /**
     * 解析 json
     *
     * @param jsonData
     * @return
     */
    public static JSONObject jsonObject(String jsonData) {
        if (StringUtils.isBlank(jsonData))
            return null;
        return JSON.parseObject(jsonData);
    }

    /**
     * 解析 json
     *
     * @param jsonData
     * @param ruleStr  规则
     * @return
     */
    public static JSONArray jsonArray(String jsonData, String ruleStr) {
        if (StringUtils.isBlank(jsonData) || StringUtils.isBlank(ruleStr))
            return null;
        String[] rules = ruleStr.split(" ");
        int ruleLen = rules.length;
        JSONArray result = null;
        JSONObject jsonObject = null;
        try {
            jsonObject = JSON.parseObject(jsonData);
        } catch (Exception e) {
            if (e instanceof ClassCastException) {
                result = JSON.parseArray(jsonData);
                return result;
            }
            logger.warn(jsonData + " jsonArray解析失败, 规则：" + ruleStr, e);
        }
        String rule = null;
        result = null;
        int getArrayIndex = 0;
        for (int i = 1; i <= ruleLen; i++) {
            rule = rules[i - 1];
            if (StringUtils.contains(rule, ":")) {//兼容数组
                String[] arr = StringUtils.split(rule, ":");
                rule = arr[0];
                if (NumberUtils.isDigits(arr[1])) {
                    getArrayIndex = Integer.parseInt(arr[1]);
                }
            }
            if (ruleLen == i) {
                result = jsonObject.getJSONArray(rule);
            } else {
                try {
                    jsonObject = jsonObject.getJSONObject(rule);
                } catch (Exception e) {
                    if (e instanceof ClassCastException) {
                        //兼容 数组
                        result = jsonObject.getJSONArray(rule);
                        jsonObject = result.getJSONObject(getArrayIndex);
                        getArrayIndex = 0;
                    } else {
                        logger.warn("解析json失败", e);
                    }
                }
            }
        }
        if (result == null)
            logger.warn("getJsonArray 解析失败, 规则：" + ruleStr);

        return result;
    }

    /**
     * 解析 json
     *
     * @param jsonData
     * @param ruleStr  规则
     * @return
     */
    public static JSONObject jsonObject(String jsonData, String ruleStr) {
        if (StringUtils.isBlank(jsonData) || StringUtils.isBlank(ruleStr))
            return null;
        String[] rules = ruleStr.split(" ");
        int ruleLen = rules.length;
        JSONObject jsonObject = JSON.parseObject(jsonData);
        String rule = null;
        JSONObject result = null;
        for (int i = 1; i <= ruleLen; i++) {
            rule = rules[i - 1];
            result = jsonObject.getJSONObject(rule);
        }
        if (result == null)
            logger.warn("jsonObject 解析失败, 规则：" + ruleStr);
        return result;
    }

    /**
     * 截取 字符串
     * @param content
     * @param ruleStr           规则  多个###隔开 ： 定位字符+"|"+截取长度+"|"+起始字符
     * @param detailSqlField
     */
    public static String substring(String content, String ruleStr, String detailSqlField) {
        if (StringUtils.isBlank(content) || StringUtils.isBlank(ruleStr))
            return null;
        content = StringEscapeUtils.unescapeHtml3(content);
        String[] rules = StringUtils.split(ruleStr, "###");//多个
        String value = null;
        for (String rule : rules) {
            value = substringStr(content, rule);
            if (StringUtils.isNotBlank(value)) {
                break;
            }
        }
        return value;
    }


    /**
     * 截取 字符串
     *
     * @param content
     * @param ruleStr 规则 subSearchText + "|" + subLength + "|" + subSearchTextIndexs
     *                定位字符       + "|" + 截取长度  +  "|" +  定位字符出现次数(多个逗号隔开)
     * @return
     */
    private static String substringStr(String content, String ruleStr) {
        String[] params = ruleStr.split("\\|");
        if (params.length < 2) {
//            logger.warn(ruleStr+" substringStr参数不齐.");
            return null;
        }
        String value = null;
        String subSearchText = params[0];
        int subLength = Integer.parseInt(params[1]);
        String subSearchTextIndexs = null;
        if (params.length > 2) {
            subSearchTextIndexs = params[2];
        }

        //定位字符出现次数(多个逗号隔开)
        if (StringUtils.isNotBlank(subSearchTextIndexs)) {
            String[] indexArray = subSearchTextIndexs.split(",");
            StringBuilder sb = new StringBuilder(32);
            int ordinal = 1;
            for (String index : indexArray) {
                try {
                    ordinal = Integer.parseInt(index);
                } catch (NumberFormatException e) {
                    throw new IllegalArgumentException("非法参数：定位字符出现次数 " + index + " 非数字");
                }
                int start = StringUtils.ordinalIndexOf(content, subSearchText, ordinal);
                if (start != -1) {
                    start += subSearchText.length();
                    value = subContent(content, start, subLength);
                    sb.append(value).append("|");
                }
            }
            value = sb.toString();
        } else {
            //定位字符
            int searchIndex = StringUtils.indexOf(content, subSearchText);
            if (searchIndex == -1) {
//                logger.warn("不存在-定位字符："+subSearchText);
                return null;
            }
            searchIndex += subSearchText.length();
            value = subContent(content, searchIndex, subLength);
        }
//        if(StringUtils.isBlank(value))
//            logger.warn("substring 解析失败, 规则："+ruleStr);
        return value;
    }

    /**
     * 截取内容
     *
     * @param content
     * @param start
     * @param subLength
     * @return
     */
    public static String subContent(String content, int start, int subLength) {
        if (StringUtils.isBlank(content)) {
            return null;
        }
        StringBuilder sb = new StringBuilder(subLength);
        final char[] chars = content.toCharArray();
        int count = 0;
        boolean ignoreChar = false;
        boolean transferChar = false;
        for (int i = start; i < chars.length && count < subLength; i++) {
            if (chars[i] == ' ')
                continue;
            if (chars[i] == '&')//含转义字符
                transferChar = true;
            if (chars[i] == '<') {
                ignoreChar = true;
                continue;
            } else if (chars[i] == '>') {
                ignoreChar = false;
                continue;
            }
            if (!ignoreChar) {
                count++;
                sb.append(chars[i]);
                //去掉网页空格
                int blankIndex = sb.indexOf(SpiderConsts.HTML_BLANK);
                if (transferChar && blankIndex != -1) {
                    sb.delete(blankIndex, sb.length());
                    count -= SpiderConsts.HTML_BLANK.length();
                    transferChar = false;
                }
            }
        }
        return sb.toString();
    }


    private static String getFileContent(String fileName) throws Exception {
        File file = new File("./" + fileName);
//        File file = new File("D:\\git\\fast_spider\\" + fileName);
        return FileUtil.readableFileSize(file);
    }

    public static void main(String[] args) throws Exception {
//        testJsoup();
//        testSubstring();
//        testJson();

//        String jsonData = getFileContent("testJson.txt");
//        String array = jsonVal(jsonData, "M_ID");
//        println(array);
//        testHtmlTable();
//        System.out.println(testJsoupDeepRule(getFileContent("testHtml.txt")));

        String url = "http://jsy.xjjs.gov.cn/dataservice/query/comp/compDetail/160418124605687189";
        Document doc = Jsoup.parse(new URL(url), 6000);
//        System.out.println(doc.html());
        String html = Jsoup.clean(doc.html(), JSOUP_WHITE_LIST_RELAXED);
        System.out.println(html);

    }



    private static void testJsoup() throws Exception {
//        Document doc = Jsoup.parse(new URL("http://www.jb51.net/article/43485.htm"), 200);
//        String title = jsoup(doc.html(), "#contents .main .article-content .title h1");
//        String attr = jsoup(doc.html(), "#head > div:nth-child(2) > div.subnav > ul > li:nth-child(6) > a@title");
//        System.out.println("title:" + title);
//        System.out.println("attr:" + attr);
        String html = getFileContent("testHtml.txt");
        System.out.println(html.length());
//        String rule = "form > table >tbody > tr > td >div > div:nth-child(3) tbody";
//        Elements elements = jsoupReturnElements(html, rule);
//        Iterator<Element> iter = elements.iterator();
//        while (iter.hasNext()) {
//            Element contentEle = iter.next();
//            Elements rows = contentEle.children();
//            Map fields = null;
//            for (Element rowEle : rows) {
////                System.out.println("发布时间：" + jsoup(rowEle.html(), "td:eq(3)"/*, fieldRuleBo.getDetailSqlField(), rinseMark, parseSuccessLogSb*/));
//                System.out.println("标题：" + jsoup(rowEle.toString(), "td:nth-child(4)"/*, fieldRuleBo.getDetailSqlField(), rinseMark, parseSuccessLogSb*/));
//                System.out.println("url：" + jsoup(rowEle.toString(), "td:nth-child(2) a@href#/Jsjzyxyglpt/faces/public/{0}"/*, fieldRuleBo.getDetailSqlField(), rinseMark, parseSuccessLogSb*/));
//            }
//        }
    }

    /**
     * 测试 Html深层次规则
     *
     * @param html
     * @return
     */
    public static String testJsoupDeepRule(String html) {
        Document doc = Jsoup.parse(html);
        Element ele = doc.body();
        List<Node> nodes = ele.childNodes();
        StringBuffer str = new StringBuffer(1024 * 15);
        recursion(nodes, ele.tagName(), str);
        return str.toString();
    }

    private static void recursion(List<Node> nodes, String pTag, StringBuffer str) {
        for (Node node : nodes) {
            str.append("规则：" + pTag + " > " + node.nodeName() + "\r\n");
            if (pTag.endsWith("li") || pTag.endsWith("td")) {
                str.append(node.toString() + "\r\n");
            }
            str.append("\r\n");
            recursion(node.childNodes(), pTag + " > " + node.nodeName(), str);
        }
    }

    private static void testSubstring() throws Exception {
        Document doc = Jsoup.parse(new URL("http://www.jb51.net/article/43485.htm"), 200);
//        String title = jsoup(doc.html(), "#contents .main .article-content .title h1", fieldRuleBo.getDetailSqlField(), rinseMark, parseSuccessLogSb);
//        String ruleStr = "转载 时间：,10";
//        String time = substring(doc.html(), ruleStr, fieldRuleBo.getDetailSqlField(), rinseMark, parseSuccessLogSb);
//        System.out.println("time:" + time);
    }

    private static void testJson() throws Exception {
        //http://sj.qq.com/myapp/searchAjax.htm?kw=%E6%89%8B%E6%9C%BA&pns=&sid=
        String url = "http://sj.qq.com/myapp/searchAjax.htm";
        Map map = new HashMap<>();
        map.put("kw", URLEncoder.encode("手机"));
        String jsonData = null;//HttpUtil.post(url, map);

        String pageNumberStack = jsonVal(jsonData, "obj pageNumberStack");
        System.out.println("pageNumberStack:" + pageNumberStack);

        JSONArray jsonArray = jsonArray(jsonData, "obj appDetails");
        for (int i = 0; i < jsonArray.size(); i++) {
            System.out.println("appName:" + jsonArray.getJSONObject(i).getString("appName"));
        }
    }

    private static void println(Object obj) {
        System.out.println(JSON.toJSONString(obj));
    }

}
